{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-09-17T04:59:22.442353Z",
     "start_time": "2025-09-17T04:59:14.631355Z"
    }
   },
   "source": [
    "import os\n",
    "os.environ[\"TAVILY_API_KEY\"] = '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'\n",
    "from llama_index.tools.tavily_research import TavilyToolSpec\n",
    "tavily_tool = TavilyToolSpec(api_key='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')\n",
    "documents = tavily_tool.search(\"好笑的笑话\", max_results=20)"
   ],
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T05:00:05.719114Z",
     "start_time": "2025-09-17T04:59:22.458357Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from llama_index.core import Settings\n",
    "os.environ['DEEPSEEK_API_KEY']='sk-f5f3487896554276aaf657fe9b9661a7'\n",
    "from llama_index.llms.deepseek import DeepSeek\n",
    "from llama_index.core.embeddings import resolve_embed_model\n",
    "\n",
    "base_embed_model = resolve_embed_model(\"local:D:/pythonProject17/transformers/model_em/BAAI/bge-small-en-v1.5\")\n",
    "llm=DeepSeek(model='deepseek-chat')\n",
    "\n",
    "Settings.llm=llm\n",
    "Settings.embed_model=base_embed_model"
   ],
   "id": "cff5d32a4474a500",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T05:00:06.118819Z",
     "start_time": "2025-09-17T05:00:05.731854Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from llama_index.core.node_parser import SentenceSplitter\n",
    "spliter=SentenceSplitter(chunk_size=512, chunk_overlap=20)"
   ],
   "id": "bff26e45c645ce44",
   "outputs": [],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T05:00:09.906624Z",
     "start_time": "2025-09-17T05:00:09.880587Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import asyncio\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()"
   ],
   "id": "8ba59c9e00742f",
   "outputs": [],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-16T19:55:01.238482Z",
     "start_time": "2025-09-16T19:54:53.204627Z"
    }
   },
   "cell_type": "code",
   "source": "nodes=await spliter.aget_nodes_from_documents(documents=documents,show_progress=True)",
   "id": "95f9b5c9ec3e60e5",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parsing nodes:   0%|          | 0/17 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "a3cb28692f9c43b5ad4f8d5abfeb0abd"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-16T19:57:08.273375Z",
     "start_time": "2025-09-16T19:57:08.253901Z"
    }
   },
   "cell_type": "code",
   "source": [
    "train_nodes=nodes[:8]\n",
    "val_nodes=nodes[8:]"
   ],
   "id": "f6fc67cc1108dd8c",
   "outputs": [],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-16T20:03:22.531018Z",
     "start_time": "2025-09-16T19:59:30.749517Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from llama_index.finetuning import generate_qa_embedding_pairs\n",
    "from llama_index.core.evaluation import EmbeddingQAFinetuneDataset\n",
    "\n",
    "train_dataset = generate_qa_embedding_pairs(train_nodes,llm=llm)\n",
    "val_dataset = generate_qa_embedding_pairs(val_nodes,llm=llm)\n",
    "\n",
    "train_dataset.save_json(\"train_dataset.json\")\n",
    "val_dataset.save_json(\"val_dataset.json\")"
   ],
   "id": "7e33ee854c17521b",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▎        | 1/8 [00:10<01:16, 10.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 2/8 [00:17<00:50,  8.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 3/8 [00:22<00:35,  7.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 4/8 [00:27<00:24,  6.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▎   | 5/8 [00:34<00:19,  6.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 6/8 [00:41<00:13,  6.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 7/8 [00:48<00:06,  6.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:54<00:00,  6.84s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final dataset saved.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 8/32 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 9/32 [00:05<02:07,  5.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███▏      | 10/32 [00:12<02:15,  6.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 11/32 [00:18<02:09,  6.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 12/32 [00:24<02:02,  6.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 13/32 [00:30<01:53,  5.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 14/32 [00:35<01:45,  5.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 15/32 [00:40<01:31,  5.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 16/32 [00:45<01:26,  5.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 17/32 [00:50<01:20,  5.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▋    | 18/32 [00:55<01:10,  5.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 19/32 [01:00<01:06,  5.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▎   | 20/32 [01:07<01:07,  5.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 21/32 [01:12<01:01,  5.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 22/32 [01:18<00:56,  5.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 23/32 [01:24<00:50,  5.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 24/32 [01:28<00:42,  5.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 25/32 [01:34<00:38,  5.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████▏ | 26/32 [01:40<00:33,  5.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 27/32 [01:45<00:26,  5.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 28/32 [01:52<00:24,  6.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 29/32 [01:58<00:17,  5.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 30/32 [02:03<00:11,  5.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 31/32 [02:08<00:05,  5.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 32/32 [02:14<00:00,  5.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "33it [02:19,  5.34s/it]                        "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "34it [02:24,  5.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "35it [02:30,  5.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "36it [02:36,  5.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "37it [02:41,  5.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "38it [02:45,  5.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "39it [02:51,  5.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "40it [02:56,  5.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final dataset saved.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "execution_count": 16
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-16T20:03:36.940830Z",
     "start_time": "2025-09-16T20:03:36.912570Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# [Optional] Load\n",
    "train_dataset = EmbeddingQAFinetuneDataset.from_json(\"train_dataset.json\")\n",
    "val_dataset = EmbeddingQAFinetuneDataset.from_json(\"val_dataset.json\")"
   ],
   "id": "87b40f2c3470e728",
   "outputs": [],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-16T20:04:21.142500Z",
     "start_time": "2025-09-16T20:04:14.454430Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from llama_index.finetuning import EmbeddingAdapterFinetuneEngine\n",
    "from llama_index.core.embeddings import resolve_embed_model\n",
    "import torch\n",
    "\n",
    "\n",
    "finetune_engine = EmbeddingAdapterFinetuneEngine(\n",
    "    train_dataset,\n",
    "    base_embed_model,\n",
    "    model_output_path=\"model_output_test\",\n",
    "    # bias=True,\n",
    "    epochs=4,\n",
    "    verbose=True,\n",
    "    # optimizer_class=torch.optim.SGD,\n",
    "    # optimizer_params={\"lr\": 0.01}\n",
    ")"
   ],
   "id": "cf25428d408d3837",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:llama_index.finetuning.embeddings.adapter:Use pytorch device: cuda\n",
      "Use pytorch device: cuda\n"
     ]
    }
   ],
   "execution_count": 18
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-16T20:04:59.315932Z",
     "start_time": "2025-09-16T20:04:53.008517Z"
    }
   },
   "cell_type": "code",
   "source": "finetune_engine.finetune()",
   "id": "aee6ce4e21c6ef98",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001B[1;3;34m> Prepared optimizer, scheduler, and loss model.\n",
      "\u001B[0m"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Epoch:   0%|          | 0/4 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "d11798b6c9e647ac96d7a20b72083205"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "data": {
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "8582882fe0814312a78af597ba12c68d"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001B[1;3;34m> [Epoch 0] Current loss: 2.039762020111084\n",
      "\u001B[0m\u001B[1;3;34m> [Epoch 0] Current loss: 1.5039902925491333\n",
      "\u001B[0m"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "92fe23f1389e4bc19fbde304d51f9e98"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001B[1;3;34m> [Epoch 1] Current loss: 2.0266976356506348\n",
      "\u001B[0m\u001B[1;3;34m> [Epoch 1] Current loss: 1.4949628114700317\n",
      "\u001B[0m"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "3ff1a56e05b54e03995b248ac2906c7f"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001B[1;3;34m> [Epoch 2] Current loss: 2.0190157890319824\n",
      "\u001B[0m\u001B[1;3;34m> [Epoch 2] Current loss: 1.4884132146835327\n",
      "\u001B[0m"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "59afcf946bbb45629762aee480dac8ad"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001B[1;3;34m> [Epoch 3] Current loss: 2.0143308639526367\n",
      "\u001B[0m\u001B[1;3;34m> [Epoch 3] Current loss: 1.4846935272216797\n",
      "\u001B[0m\u001B[1;3;34m> Finished training, saving to model_output_test\n",
      "\u001B[0m"
     ]
    }
   ],
   "execution_count": 19
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T05:00:19.005460Z",
     "start_time": "2025-09-17T05:00:18.060825Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# embed_model = finetune_engine.get_finetuned_model()\n",
    "\n",
    "# alternatively import model\n",
    "from llama_index.core.embeddings import LinearAdapterEmbeddingModel\n",
    "\n",
    "embed_model = LinearAdapterEmbeddingModel(base_embed_model, \"model_output_test\")"
   ],
   "id": "fa82b39dac58b0b7",
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "LinearAdapterEmbeddingModel() takes no arguments",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[8], line 6\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[38;5;66;03m# embed_model = finetune_engine.get_finetuned_model()\u001B[39;00m\n\u001B[0;32m      2\u001B[0m \n\u001B[0;32m      3\u001B[0m \u001B[38;5;66;03m# alternatively import model\u001B[39;00m\n\u001B[0;32m      4\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mllama_index\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mcore\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01membeddings\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m LinearAdapterEmbeddingModel\n\u001B[1;32m----> 6\u001B[0m embed_model \u001B[38;5;241m=\u001B[39m \u001B[43mLinearAdapterEmbeddingModel\u001B[49m\u001B[43m(\u001B[49m\u001B[43mbase_embed_model\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mmodel_output_test\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\n",
      "\u001B[1;31mTypeError\u001B[0m: LinearAdapterEmbeddingModel() takes no arguments"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T06:07:00.428493Z",
     "start_time": "2025-09-17T06:06:47.170304Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.core.schema import TextNode\n",
    "from tqdm.notebook import tqdm\n",
    "import pandas as pd\n",
    "\n"
   ],
   "id": "dba5a041078d5005",
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'eval_utils'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mModuleNotFoundError\u001B[0m                       Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[1], line 7\u001B[0m\n\u001B[0;32m      4\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtqdm\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mnotebook\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m tqdm\n\u001B[0;32m      5\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mpandas\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mas\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mpd\u001B[39;00m\n\u001B[1;32m----> 7\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01meval_utils\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m evaluate, display_results\n",
      "\u001B[1;31mModuleNotFoundError\u001B[0m: No module named 'eval_utils'"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "d6fc6dfbd0f38851"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
